package com.zju.cst.classification.utils;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.ansj.domain.Term;
import org.ansj.recognition.NatureRecognition;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.ansj.util.FilterModifWord;
import org.apache.log4j.Logger;

import com.zju.cst.spider.utils.StreamUtils;

public class AnsJSplit implements SplitMethod {
	private final static Logger LOG = Logger.getLogger(AnsJSplit.class);
	@Override
	public List<String> getSplit(String content) {
		LOG.debug("进入分词程序!");
		List<String> result = new ArrayList<String>();
		List<Term> list = ToAnalysis.parse(content);
		//LOG.debug("初步分词结果：");
		//list.forEach(word -> LOG.debug(word.getName()));
		new NatureRecognition(list).recognition();
		FilterModifWord.insertStopWords(StreamUtils.getStopWord("dictionary.txt"));
		list = FilterModifWord.modifResult(list);
		for (Iterator<Term> iterator = list.iterator(); iterator.hasNext();) {
			Term term = (Term) iterator.next();
			if(term.getName().matches("(.*\\d+?.*)||([\u2E80-\uFE4F])||(\\s*)")) iterator.remove();
		}
		LOG.debug("分词结束");
		list.forEach(term -> result.add(term.getName()));
		return result;
	}

}
